package com.dm.webCrawler.project.buss.service;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.net.OkHttpRequester;
import cn.edu.hfut.dmic.webcollector.plugin.rocks.BreadthCrawler;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.ZipUtil;
import cn.hutool.http.HttpRequest;
import com.dm.webCrawler.common.utils.AsponseUtil;
import com.dm.webCrawler.common.utils.StringUtils;
import com.dm.webCrawler.common.utils.file.FileUtils;
import com.dm.webCrawler.common.utils.file.XmlDetailUtil;
import com.dm.webCrawler.common.utils.http.HttpHelper;
import com.dm.webCrawler.common.utils.sign.Base64;
import com.dm.webCrawler.project.mapper.CrawlerIndexMapper;
import com.dm.webCrawler.project.mode.ZipDetialModel;
import okhttp3.Request;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.IOException;
import java.util.*;

import static sun.plugin2.os.windows.FLASHWINFO.size;

/**
 * 简式周刊内容抽取
 */
@Component
public class JanesCrawler extends BreadthCrawler {
    @Value("${lb.work-back:news}")
    private String workBack="news";
    @Value("${lb.out-zip-path:outZipPath}")
    private String outZipPath="outZipPath";
    private String baseDir;
    private String typeName="janes";
    private String start="https://www.janes.com/defence-news/";
    private String urlBase="https://www.janes.com";
    List<String> seeds=new ArrayList<String>(){{
        add(start);
//        add(start+"news/india/");
    }};

    @Autowired
    private CrawlerIndexMapper crawlerIndexMapper;

    public JanesCrawler(){
        this("JanesCrawler",true);
    }
    /**
     * 构造一个基于RocksDB的爬虫
     * RocksDB文件夹为crawlPath，crawlPath中维护了历史URL等信息
     * 不同任务不要使用相同的crawlPath
     * 两个使用相同crawlPath的爬虫并行爬取会产生错误
     *
     * @param crawlPath RocksDB使用的文件夹
     * @param autoParse 是否根据设置的正则自动探测新URL
     */

    public JanesCrawler(String crawlPath, boolean autoParse) {
        super(crawlPath, autoParse);
//        setRequester(new IndiatimesRequester()); //此处有问题
        for (String seed : seeds) {
            this.addSeed(seed);
        }
//        this.addRegex("https://www.janes.com/defence-news/news-detail/[\\s\\S]+");
//        this.addRegex("https://www.janes.com/[\\s\\S]*?-news-list");
        this.addRegex("https://www.janes.com/[\\s\\S]+");
        setThreads(10);
        getConf().setTopN(10000);
        baseDir=workBack+"/"+typeName;

    }


    @Override
    public void visit(Page page, CrawlDatums next) {

//        System.out.println(page.url());
//        if(seeds.contains(page.url())){
//            return;
//        }
//        if(page.url().startsWith("https://www.janes.com/[\\s\\S]*?-news-list")){
//            System.out.println(page.url()); ; //过滤视频
//        }
//        if(!page.url().matches("https://www.indiatimes.com/[\\s\\S]*?/[\\s\\S]*?/[\\s\\S]+\\d+.html")){
//            return;
//        }
        if(page.url().matches("https://www.janes.com/defence-news/news-detail/[\\s\\S]+")){

                printDatas(page,next);

        }

    }

    private void printDatas(Page page, CrawlDatums next){

        ZipDetialModel model=new ZipDetialModel();//输出
        model.setHqsj(DateUtil.format(new Date(),"yyyy-MM-dd HH:mm:ss"));
        String url=page.url();
        String id= StringUtils.regexFind(url,"https://www.janes.com/defence-news/news-detail/([\\s\\S]+)",1);
        id=FileUtils.titleToFileName(id);
        if(crawlerIndexMapper.hasIndex(id,typeName)>=1){
            return;
        }
        model.setWybs(id);
        String thisnewBaseDir=baseDir+"/"+id;
        new File(thisnewBaseDir).mkdirs();
        model.setLy("简式防务周刊");
        model.setFbmt("简式防务周刊");
        model.setYz("英文");
        model.setGj("美国");
        model.setDq("北美");
        model.setLydz(url);
        Elements dateAndAuthor = page.select("#Contentplaceholder1_T478E928A005_Col00 p.text-muted.small");
        //编辑人
        String bjr=dateAndAuthor.size()>=2?dateAndAuthor.get(1).text().replace("by ",""):null;
        model.setBjr(bjr);

        String bt = page.select("#Contentplaceholder1_T478E928A005_Col00 h1[itemprop='name']").first().text();
        model.setBt(bt);
        String dateStr=dateAndAuthor.first().text();
        try {
            model.setFbsj(DateUtil.format(timeStrConvert(dateStr.toUpperCase()),"yyyy-MM-dd HH:mm:ss"));
        }catch (Exception e){
            e.printStackTrace();
            System.out.println("获取发布时间失败!");
        }
        //正文文件
        page.select("#Contentplaceholder1_T478E928A005_Col00 div.mt-5").remove();
        page.select("#Contentplaceholder1_T478E928A005_Col00 div.d-none").remove();
        page.select("#Contentplaceholder1_T478E928A005_Col00 script").remove();
        String contentHtml=page.select("#Contentplaceholder1_T478E928A005_Col00",0).html();
        File wordFile = buildWordFile(contentHtml, thisnewBaseDir, id);
        //添加正文文件
        model.getWJ().add(wordFile.getName());

//        try {
//            //翻译文件
//            String chineseStr=page.select("#article-description-0",0).text();
//            String chineseFilename=id + "_zh_CN.txt";
//            File chineseFile=new File(thisnewBaseDir + "/" +chineseFilename);
//            chineseFile.createNewFile();
//            FileUtil.writeString(chineseStr,chineseFile,"UTF-8");
//            model.getZwwjlb().add(new ZipDetialModel.ZwwjlbClass(chineseFilename));
//        } catch (IOException e) {
//            e.printStackTrace();
//            System.out.println("生成翻译文件失败！");
//        }

        //输出xml文件
        try {
            XmlDetailUtil.createXML(model,thisnewBaseDir+"/detail.xml");
            //输出压缩文件
            String outPath=outZipPath+"/"+typeName+"_"+id+".zip";
            String basDirAbs=new File(thisnewBaseDir).getAbsolutePath();
            ZipUtil.zip(basDirAbs,new File(outPath).getAbsolutePath());
            FileUtil.del(basDirAbs);
            crawlerIndexMapper.appendIndex(id,typeName);
        } catch (JAXBException e) {
            e.printStackTrace();
            System.out.println("创建压缩文件失败");
        }

    }

    /**
     * 构建word文件
     * @param contentHtml
     * @param thisnewBaseDir
     * @param id
     * @return
     */
    private File buildWordFile(String contentHtml,String thisnewBaseDir,String id){
        File htmlPath = new File(thisnewBaseDir + "/" + id + ".html");
        try {
            htmlPath.createNewFile();
        } catch (IOException e) {
            e.printStackTrace();
            System.out.println("创建html文件失败！");
        }
        Document doc = Jsoup.parse(contentHtml);
        Elements imgs =doc.select("img[src]");
        for (int i = 0; i < imgs.size(); i++) {
            Element img = imgs.get(i);
            try{
                img.attr("src","data:image/png;base64,"+Base64.encode(HttpRequest.get(urlBase + img.attr("src")).execute().bodyBytes()));
            }catch (Exception e){
                e.printStackTrace();
                System.out.println("将图片转换为base64失败!"+img.attr("src"));
            }

        }
        contentHtml="<html lang=\"en\"><head></head><body>"+doc.outerHtml()+"</body></html>";
        FileUtil.writeString(contentHtml,htmlPath.getAbsolutePath(),"UTF-8");
//        System.out.println(contentHtml);
        String wordFilename=id + ".doc";
        String wordPath=thisnewBaseDir + "/" + wordFilename;
        try {
            AsponseUtil.html2word(htmlPath.toString(),wordPath);
            htmlPath.delete();
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("创建word文档失败！");
        }
        return new File(wordPath);
    }

    Map<String,String> monthConvert=new HashMap<String, String>(){{
        put("JANUARY","01");
        put("FEBRUARY","02");
        put("MARCH","03");
        put("APRIL","04");
        put("MAY","05");
        put("JUNE","06");
        put("JULY","07");
        put("AUGUST","08");
        put("SEPTEMBER","09");
        put("OCTOBER","10");
        put("NOVEMBER","11");
        put("DECEMBER","12");
    }};
    private Date timeStrConvert(String timeString){
        for (String enName : monthConvert.keySet()) {
            timeString=timeString.replace(enName,monthConvert.get(enName));
        }
        return new Date(DateUtil.parse(timeString,"dd MM yyyy").getTime()+(12*60*60*1000));
    }

    public class IndiatimesRequester extends OkHttpRequester{
        @Override
        public Request.Builder createRequestBuilder(CrawlDatum crawlDatum) {
            Request.Builder requestr = super.createRequestBuilder(crawlDatum);
            Map<String, String> headers = headerMap();
            for (String key : headers.keySet()) {
                requestr.addHeader(key,headers.get(key));
            }
            return requestr;
        }
        private Map<String,String> headerMap(){
            String headerString=":authority: www.indiatimes.com\n" +
                    ":method: GET\n" +
                    ":scheme: https\n" +
                    "accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\n" +
                    "accept-encoding: gzip, deflate, br\n" +
                    "accept-language: zh-CN,zh;q=0.9\n" +
                    "cache-control: max-age=0\n" +
                    "sec-ch-ua: \"Google Chrome\";v=\"89\", \"Chromium\";v=\"89\", \";Not A Brand\";v=\"99\"\n" +
                    "sec-ch-ua-mobile: ?0\n" +
                    "sec-fetch-dest: document\n" +
                    "sec-fetch-mode: navigate\n" +
                    "sec-fetch-site: none\n" +
                    "sec-fetch-user: ?1\n" +
                    "upgrade-insecure-requests: 1\n" +
                    "user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36";
            return HttpHelper.convertHeaderMap(headerString);
        }
    }
    
    
    public static void main(String[] args) {
        JanesCrawler crawler=new JanesCrawler("IndiatimesCrawler", true);
        try {
            crawler.start(4);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
